"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 极致RTO-数据库系统类
Case Name   : 开启极致RTO，备机进行failover过程中，备机gs_om stop，关闭极致RTO，主备数据一致
Create At   : 2020/10/15
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、备机进行failover
    3、数据库进行gs_om stop
    4、验证failover结果
    5、重启主备
    6、备机再次进行failover
    7、还原主备
    8、执行完成后，关闭极致RTO
    9、查询主备数据是否一致
Expect      :
    1、开启极致RTO成功
    2、备机开始进行failover
    3、数据库进行gs_om stop成功
    4、验证failover结果失败
    5、重启主备，恢复环境成功
    6、备机再次进行failover成功
    7、还原主备成功
    8、执行完成后，关闭极致RTO成功
    9、查询主备数据一致
History     :
    modified：2021-9-13 by 5328113;注入om stop故障偶现failover成功,优化成功后业务操作逻辑
    modified：2021-9-3 by 5328113;注入om stop故障偶现failover成功（failover成功后才stop成功）
    优化脚本逻辑，failover成功也是正常预期；
    modified：2021-4-22 by 5328113;状态结果查询方式变更，修改脚本；
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-3-5 by 5328113;failover失败可能导致升主备机与降备主机lsn不一致，导致同步存在问题，增加恢复操作
    modified：2021-2-23 by 5328113;teardown中增加恢复主备操作
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
    modified ：2020-11-18 by @peilinqian ；完善TPCC断言；tpcc执行完成后增加等待同步
    modified：2021-1-28 by 5328113;删除start成功断言，删除不必要tpcc
"""

import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import *
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info('----Opengauss_Reliability_RTO_Dbsys_Case033:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            step_txt = '----step1: 开启极致RTO，expect: 成功----'
            self.log.info(step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg, '执行失败:' + step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg, '执行失败:' + step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg, '执行失败:' + step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg, '执行失败:' + step_txt)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主备是否同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status, '执行失败:' + step_txt)

            self.log.info('----2:备机进行failover----')
            step_txt = '----step2.1:主机数据库进行gs_ctl stop；expect: 成功----'
            self.log.info(step_txt)
            stop_msg = self.pri_sh.stopDbInstance()
            self.assertIn(self.constant.GS_CTL_STOP_SUCCESS_MSG, stop_msg,
                          '执行失败:' + step_txt)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            step_txt = '----step2.2：备机进行failover，expect: 成功或失败----'
            self.log.info(step_txt)
            success_msg = self.constant.FAILOVER_SUCCESS_MSG
            failover_thread = ComThread(self.sta_1_sh.executeGsctl,
                                        args=('failover',
                                              success_msg,
                                              '',
                                              False))
            failover_thread.setDaemon(True)
            failover_thread.start()

            step_txt = '----step3: 数据库进行gs_om stop；expect: 成功----'
            self.log.info(step_txt)
            is_stop = self.pri_sh.stopDbCluster()
            self.assertTrue(is_stop, '执行失败:' + step_txt)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            step_txt = '----step4: 验证failover结果;expect: 成功或失败----'
            self.log.info(step_txt)
            failover_thread.join()
            failover_result = failover_thread.get_result()
            self.log.info(failover_result)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            step_txt = '----step5:重启主备；expect:恢复主备成功----'
            self.log.info(step_txt)
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----验证备机是否需要重建----')
            suc_flag = self.constant.REBUILD_SUCCESS_MSG
            if self.pri_sh.check_whether_need_build():
                self.log.info('--备机无法恢复，需要重建备机;expect:成功--')
                self.sta_1_sh.executeGsctl('build', suc_flag)
                self.sta_2_sh.executeGsctl('build', suc_flag)
                self.log.info('----查看主机query，同步是否正常----')
                self.pri_sh.check_location_consistency('primary',
                                                       self.node_num,
                                                       self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status, '执行失败:' + step_txt)

            step6_txt = '----step6:备机再次进行failover；----'
            self.log.info(step6_txt)
            step_txt = '----step6.1、主机数据库进行gs_ctl stop；expect:成功----'
            self.log.info(step_txt)
            stop_msg = self.pri_sh.stopDbInstance()
            self.assertIn(self.constant.GS_CTL_STOP_SUCCESS_MSG, stop_msg,
                          '执行失败:' + step_txt)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            step_txt = '----step6.2、备机再次进行failover；expect:成功----'
            self.log.info(step_txt)
            success_msg = self.constant.FAILOVER_SUCCESS_MSG
            failover_result = self.sta_1_sh.executeGsctl('failover',
                                                         success_msg,
                                                         '',
                                                         False)
            self.assertTrue(failover_result, '执行失败:' + step_txt)

            step_txt = '----step6.3:进行refresh;expect:成功----'
            self.log.info(step_txt)
            refresh_msg = self.sta_1_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg, '执行失败:' + step_txt)

            step_txt = '----step6.4、原主机以standby启动;expect:成功----'
            self.log.info(step_txt)
            start_msg = self.pri_sh.startDbInstance('standby')
            self.assertIn(self.constant.RESTART_SUCCESS_MSG, start_msg,
                          '执行失败:' + step_txt)

            self.log.info('----查看主机query，同步是否正常----')
            self.sta_1_sh.check_location_consistency('primary',
                                                     self.node_num,
                                                     self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status, '执行失败:' + step_txt)

            step7_txt = '----step7:还原主备;expect:成功----'
            self.log.info(step7_txt)
            step_txt = '----step7.1:新主机stop;expect:成功----'
            self.log.info(step_txt)
            stop_msg = self.sta_1_sh.stopDbInstance()
            self.assertIn(self.constant.GS_CTL_STOP_SUCCESS_MSG, stop_msg,
                          '执行失败:' + step_txt)

            step_txt = '----step7.2:新备机进行failover还原主备;expect:成功----'
            self.log.info(step_txt)
            success_msg = self.constant.FAILOVER_SUCCESS_MSG
            failover_result = self.pri_sh.executeGsctl('failover',
                                                       success_msg,
                                                       '',
                                                       False)
            self.assertTrue(failover_result, '执行失败:' + step_txt)

            step_txt = '----step7.3:进行refresh;expect:成功----'
            self.log.info(step_txt)
            refresh_msg = self.pri_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg, '执行失败:' + step_txt)

            step_txt = '----step7.4:新主机以standby启动;expect:成功----'
            self.log.info(step_txt)
            stop_msg = self.sta_1_sh.startDbInstance('standby')
            self.assertIn(self.constant.RESTART_SUCCESS_MSG, stop_msg,
                          '执行失败:' + step_txt)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status, '执行失败:' + step7_txt)

        finally:
            step_txt = '----step8:执行完成后，关闭极致RTO;expect:成功----'
            self.log.info(step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg, '执行失败:' + step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg, '执行失败:' + step_txt)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg, '执行失败:' + step_txt)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1, '执行失败:' + step_txt)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2, '执行失败:' + step_txt)

            step_txt = '----step9:查询主备数据是否一致;expect:成功----'
            self.log.info(step_txt)
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag, '执行失败:' + step_txt)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag, '执行失败:' + step_txt)

    def tearDown(self):
        self.log.info("----this is tearDown----")

        host_name = self.pri_root.sh('hostname').result()
        if self.pri_sh.check_whether_need_switch(host_name):
            self.log.info('----恢复主备关系----')
            if self.pri_sh.check_whether_need_build():
                self.log.info('如果备机无法恢复，需要重建备机')
                self.pri_sh.executeGsctl('build',
                                         self.constant.REBUILD_SUCCESS_MSG)
            self.log.info('---进行主备切换---')
            success_msg = self.constant.SWITCHOVER_SUCCESS_MSG
            result = self.pri_sh.executeGsctl('switchover', success_msg)
            self.log.info(result)
            result = self.pri_sh.exec_refresh_conf()
            self.log.info(result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info('----Opengauss_Reliability_RTO_Dbsys_Case033:执行完毕----')
